from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re

print "Updated by Wei on 2014/06/27"
print "STOP develop this program. The POW(pt,0), POW(pt,1) will affect the order of the posting array. So it is NOT good just modifying here"
exit(1)


print "Begins..."
docIDLowerBound = 0
docIDUpperBound = 0
numOfDocsProcessed = 0
print "len(sys.argv):",len(sys.argv)

if len(sys.argv) != 3:
    print "Illegal # of arguments"
    print "Usage: python programName.py docIDLowerBound docIDUpperBound"
    exit(1)
else:
    docIDLowerBound = int(sys.argv[1])
    docIDUpperBound = int(sys.argv[2])
    numOfDocsProcessed = docIDUpperBound - docIDLowerBound
    print "docIDLowerBound:",docIDLowerBound
    print "docIDUpperBound:",docIDUpperBound
    print "numOfDocsProcessed:",numOfDocsProcessed
    

# termID
# firstFactorProbablity
docIDBeginningPositionDict = {}
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/dynamicUnigramFromWei/documentPostingArrays/gov2/LEAVE_auxFileForNavigatingBinaryPostingFile_WHOLE"
inputFileHandler = open(inputFileName0,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentBeginningDocID = int(lineElements[0])
    currentBeginningPosition = int(lineElements[1])
    docIDBeginningPositionDict[currentBeginningDocID] = currentBeginningPosition
inputFileHandler.close()
print "len(docIDBeginningPositionDict):",len(docIDBeginningPositionDict)
print "docIDBeginningPositionDict[0]:",docIDBeginningPositionDict[0]
print "docIDBeginningPositionDict[6000000]:",docIDBeginningPositionDict[6000000]
print "docIDBeginningPositionDict[10000000]:",docIDBeginningPositionDict[10000000]



BIG_NUMBER = 100000000
# termID
# firstFactorProbablity
firstFactorProbablityDict = {}
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/LMs/GOV2/wholeLexicon_GOV2_goodTuring_unigramProbablity_20140619_SIMPLIFIED"
inputFileHandler = open(inputFileName0,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if lineElements[0] != "UNK":
        termID = int(lineElements[0])
    else:
        termID = BIG_NUMBER
    firstFactorProbability = float(lineElements[1])
    if termID not in firstFactorProbablityDict:
        firstFactorProbablityDict[termID] = firstFactorProbability 
inputFileHandler.close()
print "len(firstFactorProbablityDict):",len(firstFactorProbablityDict)
print "firstFactorProbablityDict[33254377]:",firstFactorProbablityDict[33254377]

inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/dynamicUnigramFromWei/documentPostingArrays/gov2/GOV2_DocumentPostingArray_WHOLE_20140627_pt_2DTableFromTOP100Postings.binary"
print "inputFileName1:",inputFileName1
inputFileHandler = open(inputFileName1,"rb")
statinfo = os.stat(currentInputFileName)
fileSize = statinfo.st_size
fileBeginningPosition = 0
if docIDLowerBound not in docIDBeginningPositionDict:
    exit(1)
else:
    fileBeginningPosition = docIDBeginningPositionDict[docIDLowerBound]
print "file size:",fileSize
print "fileBeginningPosition:",fileBeginningPosition
inputFileHandler.seek(fileBeginningPosition)
numOfBytesRead = 0
while numOfBytesRead < fileSize:
    byteString = inputFileHandler.read(4 + 4)
    (docID,numOfPostings) = unpack( "2I", byteString)
    xDoc = 0.0
    sumStatic0 = 0.0
    sumStatic1 = 0.0
    sumStatic2 = 0.0
    sumStatic3 = 0.0
    sumStatic4 = 0.0
    sumScore = 0.0
    for i in range(0,numOfPostings):
        byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
        (termID,static,_,_,score) = unpack( "1I4f", byteString)
        # print "----->",termID,static,score
        firstFactorProbablity = 0.0
        secondFactorProbablity = 0.0
        secondFactorProbablityPOWTo0 = 0.0
        secondFactorProbablityPOWTo1 = 0.0
        secondFactorProbablityPOWTo2 = 0.0
        secondFactorProbablityPOWTo3 = 0.0
        secondFactorProbablityPOWTo4 = 0.0
        newStatic0 = 0.0
        newStatic1 = 0.0
        newStatic2 = 0.0
        newStatic3 = 0.0
        newStatic4 = 0.0
        if termID in firstFactorProbablityDict:
            firstFactorProbablity = firstFactorProbablityDict[termID]
        else:
            firstFactorProbablity = firstFactorProbablityDict[BIG_NUMBER]
        secondFactorProbablity = static / firstFactorProbablity
        secondFactorProbablityPOWTo0 = math.pow(secondFactorProbablity, 0)
        secondFactorProbablityPOWTo1 = math.pow(secondFactorProbablity, 1)
        secondFactorProbablityPOWTo2 = math.pow(secondFactorProbablity, 0.3)
        secondFactorProbablityPOWTo3 = math.pow(secondFactorProbablity, 0.5)
        secondFactorProbablityPOWTo4 = math.pow(secondFactorProbablity, 0.7)
        newStatic0 = firstFactorProbablity * secondFactorProbablityPOWTo0
        newStatic1 = firstFactorProbablity * secondFactorProbablityPOWTo1
        newStatic2 = firstFactorProbablity * secondFactorProbablityPOWTo2
        newStatic3 = firstFactorProbablity * secondFactorProbablityPOWTo3
        newStatic4 = firstFactorProbablity * secondFactorProbablityPOWTo4
        # print termID,firstFactorProbablity,secondFactorProbablity,newStatic0,newStatic1,newStatic2,newStatic3,newStatic4,score
        xDoc += firstFactorProbablity
        sumStatic0 += newStatic0
        sumStatic1 += newStatic1
        sumStatic2 += newStatic2
        sumStatic3 += newStatic3
        sumStatic4 += newStatic4
        sumScore += score
    numOfBytesRead += 8 + numOfPostings * 4 * 5
    # inputFileHandler.seek(numOfBytesRead)
    print docID,numOfPostings,xDoc,sumStatic0,sumStatic1,sumStatic2,sumStatic3,sumStatic4,sumScore
    # if docID == 1:
    #    exit(1)
inputFileHandler.close()
print "Ends."







